{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o7H1OrBb2L6P"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2nq95nVQE_00",
        "outputId": "6257f18c-6cd7-45c6-8b11-e74d7f489e1d"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cuda', index=0)"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "# defining our device, 'cuda:0' if CUDA is available, 'cpu' otherwise\n",
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
        "device"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4RlJUUwp2NnJ"
      },
      "outputs": [],
      "source": [
        "# make the transform pipeline, converting to tensor and normalizing\n",
        "transform = transforms.Compose([\n",
        "  transforms.ToTensor(),\n",
        "  transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
        "])\n",
        "# the batch size during training\n",
        "batch_size = 64"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86,
          "referenced_widgets": [
            "b4a6874638e2443f8c43e0db744fa327",
            "befda00472714307afad3307ced59f39",
            "073e2b2d0c164e5ba3ab91c64618e27c",
            "367a2d97f6014f67b2557e5e26a76e2b",
            "faa9f040562349cea6bf0b619249d5b4",
            "506e3b6439b6476499deaf812300bf48",
            "fee449ec39d24cbdbd6912bbdd95649f",
            "ccda80a9207f470bb93b1ce9c7fb67d4",
            "ad67632afdf947c8b7136799037a63df",
            "50a6c0a6110e470291a56b008d7657b3",
            "2ec3f8f6c8ae408ea62db2cc9368483a"
          ]
        },
        "id": "7G1YZeI22dlK",
        "outputId": "1baea523-3955-445e-de6a-8a3e7ec1ebe2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/170498071 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b4a6874638e2443f8c43e0db744fa327"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./data/cifar-10-python.tar.gz to ./data\n"
          ]
        }
      ],
      "source": [
        "train_dataset = torchvision.datasets.CIFAR10(root=\"./data\", train=True, \n",
        "                                             download=True, transform=transform)\n",
        "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size,\n",
        "                                           shuffle=True, num_workers=2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mcYaFCZo2roH",
        "outputId": "600e0580-e710-4d13-e569-1f04fa158868"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Files already downloaded and verified\n"
          ]
        }
      ],
      "source": [
        "test_dataset = torchvision.datasets.CIFAR10(root=\"./data\", train=False, \n",
        "                                             download=True, transform=transform)\n",
        "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size,\n",
        "                                           shuffle=False, num_workers=2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mNpivOW92yvE"
      },
      "outputs": [],
      "source": [
        "# the MNIST classes\n",
        "classes = ('plane', 'car', 'bird', 'cat',\n",
        "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dkVdA46x3Gle",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "951fbd28-7c82-484b-934b-30a73ad3e085"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Net(\n",
              "  (conv1): Conv2d(3, 6, kernel_size=(5, 5), stride=(1, 1))\n",
              "  (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "  (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n",
              "  (fc1): Linear(in_features=400, out_features=120, bias=True)\n",
              "  (fc2): Linear(in_features=120, out_features=84, bias=True)\n",
              "  (fc3): Linear(in_features=84, out_features=10, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ],
      "source": [
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "\n",
        "class Net(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.conv1 = nn.Conv2d(3, 6, 5)\n",
        "        self.pool = nn.MaxPool2d(2, 2)\n",
        "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
        "        self.fc1 = nn.Linear(16 * 5 * 5, 120)\n",
        "        self.fc2 = nn.Linear(120, 84)\n",
        "        self.fc3 = nn.Linear(84, 10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.pool(F.relu(self.conv1(x)))\n",
        "        x = self.pool(F.relu(self.conv2(x)))\n",
        "        x = torch.flatten(x, 1) # flatten all dimensions except batch\n",
        "        x = F.relu(self.fc1(x))\n",
        "        x = F.relu(self.fc2(x))\n",
        "        x = self.fc3(x)\n",
        "        return x\n",
        "\n",
        "\n",
        "net = Net()\n",
        "# switch to GPU if available\n",
        "net.to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T1-6Azns3OJJ"
      },
      "outputs": [],
      "source": [
        "import torch.optim as optim\n",
        "\n",
        "# defining the loss and the optimizer\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "npP_BSX-C8w7"
      },
      "outputs": [],
      "source": [
        "def get_test_loss(net, criterion, data_loader):\n",
        "  \"\"\"A simple function that iterates over `data_loader` to calculate the overall loss\"\"\"\n",
        "  testing_loss = []\n",
        "  with torch.no_grad():\n",
        "    for data in data_loader:\n",
        "      inputs, labels = data\n",
        "      # get the data to GPU (if available)\n",
        "      inputs, labels = inputs.to(device), labels.to(device)\n",
        "      outputs = net(inputs)\n",
        "      # calculate the loss for this batch\n",
        "      loss = criterion(outputs, labels)\n",
        "      # add the loss of this batch to the list\n",
        "      testing_loss.append(loss.item())\n",
        "  # calculate the average loss\n",
        "  return sum(testing_loss) / len(testing_loss)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MFohsd433Qfh",
        "outputId": "3a53b354-5f73-4cd1-fb44-2a04a6c96da3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[ 1] [it= 1000] Train Loss: 2.298, Test Loss: 2.280\n",
            "[ 2] [it= 2000] Train Loss: 2.087, Test Loss: 1.907\n",
            "[ 3] [it= 3000] Train Loss: 1.785, Test Loss: 1.685\n",
            "[ 5] [it= 4000] Train Loss: 1.631, Test Loss: 1.574\n",
            "[ 6] [it= 5000] Train Loss: 1.529, Test Loss: 1.484\n",
            "[ 7] [it= 6000] Train Loss: 1.449, Test Loss: 1.421\n",
            "[ 8] [it= 7000] Train Loss: 1.392, Test Loss: 1.373\n",
            "[10] [it= 8000] Train Loss: 1.329, Test Loss: 1.316\n",
            "[11] [it= 9000] Train Loss: 1.276, Test Loss: 1.279\n",
            "[12] [it=10000] Train Loss: 1.236, Test Loss: 1.232\n",
            "[14] [it=11000] Train Loss: 1.188, Test Loss: 1.194\n",
            "[15] [it=12000] Train Loss: 1.139, Test Loss: 1.202\n",
            "[16] [it=13000] Train Loss: 1.111, Test Loss: 1.157\n",
            "[17] [it=14000] Train Loss: 1.081, Test Loss: 1.151\n",
            "[19] [it=15000] Train Loss: 1.049, Test Loss: 1.112\n",
            "[20] [it=16000] Train Loss: 1.019, Test Loss: 1.116\n",
            "[21] [it=17000] Train Loss: 1.003, Test Loss: 1.115\n",
            "[23] [it=18000] Train Loss: 0.974, Test Loss: 1.048\n",
            "[24] [it=19000] Train Loss: 0.949, Test Loss: 1.037\n",
            "[25] [it=20000] Train Loss: 0.928, Test Loss: 1.037\n",
            "[26] [it=21000] Train Loss: 0.915, Test Loss: 1.042\n",
            "[28] [it=22000] Train Loss: 0.894, Test Loss: 1.052\n",
            "[29] [it=23000] Train Loss: 0.871, Test Loss: 1.023\n",
            "[30] [it=24000] Train Loss: 0.857, Test Loss: 1.029\n",
            "[31] [it=25000] Train Loss: 0.843, Test Loss: 1.011\n",
            "[33] [it=26000] Train Loss: 0.819, Test Loss: 1.025\n",
            "[34] [it=27000] Train Loss: 0.803, Test Loss: 1.010\n",
            "[35] [it=28000] Train Loss: 0.792, Test Loss: 1.005\n",
            "[37] [it=29000] Train Loss: 0.780, Test Loss: 0.993\n",
            "[38] [it=30000] Train Loss: 0.755, Test Loss: 1.000\n",
            "[39] [it=31000] Train Loss: 0.747, Test Loss: 1.021\n",
            "[40] [it=32000] Train Loss: 0.734, Test Loss: 1.002\n",
            "[42] [it=33000] Train Loss: 0.714, Test Loss: 1.069\n",
            "[43] [it=34000] Train Loss: 0.701, Test Loss: 1.026\n",
            "[44] [it=35000] Train Loss: 0.694, Test Loss: 1.039\n",
            "[46] [it=36000] Train Loss: 0.677, Test Loss: 1.044\n",
            "[47] [it=37000] Train Loss: 0.662, Test Loss: 1.045\n",
            "[48] [it=38000] Train Loss: 0.648, Test Loss: 1.064\n",
            "[49] [it=39000] Train Loss: 0.637, Test Loss: 1.040\n",
            "[51] [it=40000] Train Loss: 0.623, Test Loss: 1.059\n",
            "[52] [it=41000] Train Loss: 0.618, Test Loss: 1.062\n",
            "[53] [it=42000] Train Loss: 0.602, Test Loss: 1.081\n",
            "[54] [it=43000] Train Loss: 0.594, Test Loss: 1.081\n",
            "[56] [it=44000] Train Loss: 0.570, Test Loss: 1.138\n",
            "[57] [it=45000] Train Loss: 0.566, Test Loss: 1.114\n",
            "[58] [it=46000] Train Loss: 0.557, Test Loss: 1.136\n",
            "[60] [it=47000] Train Loss: 0.540, Test Loss: 1.150\n",
            "[61] [it=48000] Train Loss: 0.531, Test Loss: 1.131\n",
            "[62] [it=49000] Train Loss: 0.522, Test Loss: 1.133\n",
            "[63] [it=50000] Train Loss: 0.511, Test Loss: 1.163\n",
            "[65] [it=51000] Train Loss: 0.498, Test Loss: 1.257\n",
            "[66] [it=52000] Train Loss: 0.488, Test Loss: 1.206\n",
            "[67] [it=53000] Train Loss: 0.480, Test Loss: 1.218\n",
            "[69] [it=54000] Train Loss: 0.467, Test Loss: 1.257\n",
            "[70] [it=55000] Train Loss: 0.448, Test Loss: 1.274\n",
            "[71] [it=56000] Train Loss: 0.449, Test Loss: 1.332\n",
            "[72] [it=57000] Train Loss: 0.434, Test Loss: 1.281\n",
            "[74] [it=58000] Train Loss: 0.429, Test Loss: 1.337\n",
            "[75] [it=59000] Train Loss: 0.411, Test Loss: 1.358\n",
            "[76] [it=60000] Train Loss: 0.410, Test Loss: 1.385\n",
            "[78] [it=61000] Train Loss: 0.407, Test Loss: 1.394\n",
            "[79] [it=62000] Train Loss: 0.387, Test Loss: 1.432\n",
            "[80] [it=63000] Train Loss: 0.383, Test Loss: 1.421\n",
            "[81] [it=64000] Train Loss: 0.372, Test Loss: 1.460\n",
            "[83] [it=65000] Train Loss: 0.366, Test Loss: 1.476\n",
            "[84] [it=66000] Train Loss: 0.348, Test Loss: 1.534\n",
            "[85] [it=67000] Train Loss: 0.347, Test Loss: 1.561\n",
            "[86] [it=68000] Train Loss: 0.341, Test Loss: 1.562\n",
            "[88] [it=69000] Train Loss: 0.328, Test Loss: 1.613\n",
            "[89] [it=70000] Train Loss: 0.324, Test Loss: 1.618\n",
            "[90] [it=71000] Train Loss: 0.315, Test Loss: 1.669\n",
            "[92] [it=72000] Train Loss: 0.311, Test Loss: 1.705\n",
            "[93] [it=73000] Train Loss: 0.291, Test Loss: 1.756\n",
            "[94] [it=74000] Train Loss: 0.291, Test Loss: 1.816\n",
            "[95] [it=75000] Train Loss: 0.288, Test Loss: 1.785\n",
            "[97] [it=76000] Train Loss: 0.283, Test Loss: 1.820\n",
            "[98] [it=77000] Train Loss: 0.266, Test Loss: 1.890\n",
            "[99] [it=78000] Train Loss: 0.262, Test Loss: 1.961\n",
            "[101] [it=79000] Train Loss: 0.261, Test Loss: 1.906\n",
            "[102] [it=80000] Train Loss: 0.242, Test Loss: 1.993\n",
            "[103] [it=81000] Train Loss: 0.236, Test Loss: 2.001\n",
            "[104] [it=82000] Train Loss: 0.244, Test Loss: 2.063\n",
            "[106] [it=83000] Train Loss: 0.230, Test Loss: 2.062\n",
            "[107] [it=84000] Train Loss: 0.221, Test Loss: 2.186\n",
            "[108] [it=85000] Train Loss: 0.230, Test Loss: 2.168\n",
            "[109] [it=86000] Train Loss: 0.225, Test Loss: 2.158\n",
            "[111] [it=87000] Train Loss: 0.209, Test Loss: 2.246\n",
            "[112] [it=88000] Train Loss: 0.197, Test Loss: 2.280\n",
            "[113] [it=89000] Train Loss: 0.204, Test Loss: 2.338\n",
            "[115] [it=90000] Train Loss: 0.193, Test Loss: 2.355\n",
            "[116] [it=91000] Train Loss: 0.190, Test Loss: 2.422\n",
            "[117] [it=92000] Train Loss: 0.185, Test Loss: 2.453\n",
            "[118] [it=93000] Train Loss: 0.192, Test Loss: 2.468\n",
            "[120] [it=94000] Train Loss: 0.180, Test Loss: 2.537\n",
            "[121] [it=95000] Train Loss: 0.165, Test Loss: 2.612\n",
            "[122] [it=96000] Train Loss: 0.172, Test Loss: 2.609\n",
            "[124] [it=97000] Train Loss: 0.176, Test Loss: 2.651\n",
            "[125] [it=98000] Train Loss: 0.173, Test Loss: 2.635\n",
            "[126] [it=99000] Train Loss: 0.165, Test Loss: 2.786\n",
            "[127] [it=100000] Train Loss: 0.164, Test Loss: 2.765\n",
            "[129] [it=101000] Train Loss: 0.159, Test Loss: 2.876\n",
            "[130] [it=102000] Train Loss: 0.148, Test Loss: 2.814\n",
            "[131] [it=103000] Train Loss: 0.141, Test Loss: 2.925\n",
            "[132] [it=104000] Train Loss: 0.139, Test Loss: 2.901\n",
            "[134] [it=105000] Train Loss: 0.136, Test Loss: 3.002\n",
            "[135] [it=106000] Train Loss: 0.137, Test Loss: 3.042\n",
            "[136] [it=107000] Train Loss: 0.141, Test Loss: 3.047\n",
            "[138] [it=108000] Train Loss: 0.132, Test Loss: 3.099\n",
            "[139] [it=109000] Train Loss: 0.136, Test Loss: 3.107\n",
            "[140] [it=110000] Train Loss: 0.146, Test Loss: 3.267\n",
            "[141] [it=111000] Train Loss: 0.141, Test Loss: 3.116\n",
            "[143] [it=112000] Train Loss: 0.111, Test Loss: 3.261\n",
            "[144] [it=113000] Train Loss: 0.130, Test Loss: 3.300\n",
            "[145] [it=114000] Train Loss: 0.113, Test Loss: 3.304\n",
            "[147] [it=115000] Train Loss: 0.131, Test Loss: 3.311\n",
            "[148] [it=116000] Train Loss: 0.111, Test Loss: 3.361\n",
            "[149] [it=117000] Train Loss: 0.111, Test Loss: 3.359\n",
            "Done training.\n"
          ]
        }
      ],
      "source": [
        "training_loss, testing_loss = [], []\n",
        "running_loss = []\n",
        "i = 0\n",
        "for epoch in range(150): # 150 epochs\n",
        "  for data in train_loader:\n",
        "    inputs, labels = data\n",
        "    # get the data to GPU (if available)\n",
        "    inputs, labels = inputs.to(device), labels.to(device)\n",
        "    optimizer.zero_grad()\n",
        "    # forward pass\n",
        "    outputs = net(inputs)\n",
        "    # backward pass\n",
        "    loss = criterion(outputs, labels)\n",
        "    loss.backward()\n",
        "    # update gradients\n",
        "    optimizer.step()\n",
        "    running_loss.append(loss.item())\n",
        "    i += 1\n",
        "    if i % 1000 == 0:\n",
        "      avg_train_loss = sum(running_loss) / len(running_loss)\n",
        "      avg_test_loss = get_test_loss(net, criterion, test_loader)\n",
        "      # clear the list\n",
        "      running_loss.clear()\n",
        "      # for logging & plotting later\n",
        "      training_loss.append(avg_train_loss)\n",
        "      testing_loss.append(avg_test_loss)\n",
        "      print(f\"[{epoch:2d}] [it={i:5d}] Train Loss: {avg_train_loss:.3f}, Test Loss: {avg_test_loss:.3f}\")\n",
        "\n",
        "  \n",
        "print(\"Done training.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PvQZ2SliFsIB",
        "outputId": "968c1a5d-25f2-4490-b16b-b2eb235a57e2"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "NetDropout(\n",
              "  (conv1): Conv2d(3, 6, kernel_size=(5, 5), stride=(1, 1))\n",
              "  (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "  (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n",
              "  (do1): Dropout(p=0.2, inplace=False)\n",
              "  (fc1): Linear(in_features=400, out_features=120, bias=True)\n",
              "  (do2): Dropout(p=0.2, inplace=False)\n",
              "  (fc2): Linear(in_features=120, out_features=84, bias=True)\n",
              "  (do3): Dropout(p=0.1, inplace=False)\n",
              "  (fc3): Linear(in_features=84, out_features=10, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "class NetDropout(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.conv1 = nn.Conv2d(3, 6, 5)\n",
        "        self.pool = nn.MaxPool2d(2, 2)\n",
        "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
        "        self.do1 = nn.Dropout(0.2)  # 20% Probability\n",
        "        self.fc1 = nn.Linear(16 * 5 * 5, 120)\n",
        "        self.do2 = nn.Dropout(0.2)  # 20% Probability\n",
        "        self.fc2 = nn.Linear(120, 84)\n",
        "        self.do3 = nn.Dropout(0.1)  # 10% Probability\n",
        "        self.fc3 = nn.Linear(84, 10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.pool(F.relu(self.conv1(x)))\n",
        "        x = self.pool(F.relu(self.conv2(x)))\n",
        "        x = self.do1(x)\n",
        "        x = torch.flatten(x, 1) # flatten all dimensions except batch\n",
        "        x = F.relu(self.fc1(x))\n",
        "        x = self.do2(x)\n",
        "        x = F.relu(self.fc2(x))\n",
        "        x = self.do3(x)\n",
        "        x = self.fc3(x)\n",
        "        return x\n",
        "\n",
        "\n",
        "net_dropout = NetDropout()\n",
        "net_dropout.to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SEA_uz78LJyi"
      },
      "outputs": [],
      "source": [
        "import torch.optim as optim\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = optim.SGD(net_dropout.parameters(), lr=0.001, momentum=0.9)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GVxgRJtD55Le",
        "outputId": "d205210e-df11-4dfd-f766-f7c77445cb3c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[ 1] [it= 1000] Train Loss: 2.296, Test Loss: 2.270\n",
            "[ 2] [it= 2000] Train Loss: 2.113, Test Loss: 1.992\n",
            "[ 3] [it= 3000] Train Loss: 1.910, Test Loss: 1.810\n",
            "[ 5] [it= 4000] Train Loss: 1.757, Test Loss: 1.680\n",
            "[ 6] [it= 5000] Train Loss: 1.660, Test Loss: 1.607\n",
            "[ 7] [it= 6000] Train Loss: 1.587, Test Loss: 1.563\n",
            "[ 8] [it= 7000] Train Loss: 1.535, Test Loss: 1.499\n",
            "[10] [it= 8000] Train Loss: 1.488, Test Loss: 1.481\n",
            "[11] [it= 9000] Train Loss: 1.455, Test Loss: 1.441\n",
            "[12] [it=10000] Train Loss: 1.426, Test Loss: 1.412\n",
            "[14] [it=11000] Train Loss: 1.400, Test Loss: 1.394\n",
            "[15] [it=12000] Train Loss: 1.377, Test Loss: 1.365\n",
            "[16] [it=13000] Train Loss: 1.350, Test Loss: 1.341\n",
            "[17] [it=14000] Train Loss: 1.332, Test Loss: 1.324\n",
            "[19] [it=15000] Train Loss: 1.311, Test Loss: 1.325\n",
            "[20] [it=16000] Train Loss: 1.289, Test Loss: 1.309\n",
            "[21] [it=17000] Train Loss: 1.264, Test Loss: 1.280\n",
            "[23] [it=18000] Train Loss: 1.255, Test Loss: 1.290\n",
            "[24] [it=19000] Train Loss: 1.238, Test Loss: 1.255\n",
            "[25] [it=20000] Train Loss: 1.218, Test Loss: 1.250\n",
            "[26] [it=21000] Train Loss: 1.202, Test Loss: 1.220\n",
            "[28] [it=22000] Train Loss: 1.188, Test Loss: 1.219\n",
            "[29] [it=23000] Train Loss: 1.174, Test Loss: 1.216\n",
            "[30] [it=24000] Train Loss: 1.161, Test Loss: 1.207\n",
            "[31] [it=25000] Train Loss: 1.146, Test Loss: 1.185\n",
            "[33] [it=26000] Train Loss: 1.134, Test Loss: 1.201\n",
            "[34] [it=27000] Train Loss: 1.116, Test Loss: 1.179\n",
            "[35] [it=28000] Train Loss: 1.109, Test Loss: 1.190\n",
            "[37] [it=29000] Train Loss: 1.102, Test Loss: 1.145\n",
            "[38] [it=30000] Train Loss: 1.085, Test Loss: 1.164\n",
            "[39] [it=31000] Train Loss: 1.075, Test Loss: 1.148\n",
            "[40] [it=32000] Train Loss: 1.067, Test Loss: 1.146\n",
            "[42] [it=33000] Train Loss: 1.061, Test Loss: 1.131\n",
            "[43] [it=34000] Train Loss: 1.045, Test Loss: 1.145\n",
            "[44] [it=35000] Train Loss: 1.039, Test Loss: 1.127\n",
            "[46] [it=36000] Train Loss: 1.027, Test Loss: 1.140\n",
            "[47] [it=37000] Train Loss: 1.020, Test Loss: 1.133\n",
            "[48] [it=38000] Train Loss: 1.014, Test Loss: 1.109\n",
            "[49] [it=39000] Train Loss: 1.011, Test Loss: 1.104\n",
            "[51] [it=40000] Train Loss: 1.002, Test Loss: 1.113\n",
            "[52] [it=41000] Train Loss: 0.993, Test Loss: 1.105\n",
            "[53] [it=42000] Train Loss: 0.991, Test Loss: 1.098\n",
            "[54] [it=43000] Train Loss: 0.984, Test Loss: 1.091\n",
            "[56] [it=44000] Train Loss: 0.971, Test Loss: 1.099\n",
            "[57] [it=45000] Train Loss: 0.964, Test Loss: 1.106\n",
            "[58] [it=46000] Train Loss: 0.960, Test Loss: 1.080\n",
            "[60] [it=47000] Train Loss: 0.954, Test Loss: 1.095\n",
            "[61] [it=48000] Train Loss: 0.947, Test Loss: 1.095\n",
            "[62] [it=49000] Train Loss: 0.943, Test Loss: 1.091\n",
            "[63] [it=50000] Train Loss: 0.945, Test Loss: 1.099\n",
            "[65] [it=51000] Train Loss: 0.931, Test Loss: 1.092\n",
            "[66] [it=52000] Train Loss: 0.927, Test Loss: 1.090\n",
            "[67] [it=53000] Train Loss: 0.927, Test Loss: 1.074\n",
            "[69] [it=54000] Train Loss: 0.927, Test Loss: 1.076\n",
            "[70] [it=55000] Train Loss: 0.913, Test Loss: 1.073\n",
            "[71] [it=56000] Train Loss: 0.916, Test Loss: 1.075\n",
            "[72] [it=57000] Train Loss: 0.910, Test Loss: 1.086\n",
            "[74] [it=58000] Train Loss: 0.901, Test Loss: 1.091\n",
            "[75] [it=59000] Train Loss: 0.901, Test Loss: 1.076\n",
            "[76] [it=60000] Train Loss: 0.898, Test Loss: 1.094\n",
            "[78] [it=61000] Train Loss: 0.897, Test Loss: 1.064\n",
            "[79] [it=62000] Train Loss: 0.885, Test Loss: 1.090\n",
            "[80] [it=63000] Train Loss: 0.887, Test Loss: 1.065\n",
            "[81] [it=64000] Train Loss: 0.888, Test Loss: 1.091\n",
            "[83] [it=65000] Train Loss: 0.882, Test Loss: 1.075\n",
            "[84] [it=66000] Train Loss: 0.875, Test Loss: 1.069\n",
            "[85] [it=67000] Train Loss: 0.872, Test Loss: 1.087\n",
            "[86] [it=68000] Train Loss: 0.872, Test Loss: 1.074\n",
            "[88] [it=69000] Train Loss: 0.865, Test Loss: 1.084\n",
            "[89] [it=70000] Train Loss: 0.861, Test Loss: 1.086\n",
            "[90] [it=71000] Train Loss: 0.865, Test Loss: 1.086\n",
            "[92] [it=72000] Train Loss: 0.859, Test Loss: 1.069\n",
            "[93] [it=73000] Train Loss: 0.853, Test Loss: 1.067\n",
            "[94] [it=74000] Train Loss: 0.855, Test Loss: 1.076\n",
            "[95] [it=75000] Train Loss: 0.853, Test Loss: 1.081\n",
            "[97] [it=76000] Train Loss: 0.847, Test Loss: 1.095\n",
            "[98] [it=77000] Train Loss: 0.846, Test Loss: 1.112\n",
            "[99] [it=78000] Train Loss: 0.849, Test Loss: 1.079\n",
            "[101] [it=79000] Train Loss: 0.841, Test Loss: 1.083\n",
            "[102] [it=80000] Train Loss: 0.833, Test Loss: 1.075\n",
            "[103] [it=81000] Train Loss: 0.837, Test Loss: 1.082\n",
            "[104] [it=82000] Train Loss: 0.837, Test Loss: 1.105\n",
            "[106] [it=83000] Train Loss: 0.832, Test Loss: 1.086\n",
            "[107] [it=84000] Train Loss: 0.829, Test Loss: 1.088\n",
            "[108] [it=85000] Train Loss: 0.831, Test Loss: 1.075\n",
            "[109] [it=86000] Train Loss: 0.823, Test Loss: 1.077\n",
            "[111] [it=87000] Train Loss: 0.822, Test Loss: 1.083\n",
            "[112] [it=88000] Train Loss: 0.819, Test Loss: 1.081\n",
            "[113] [it=89000] Train Loss: 0.824, Test Loss: 1.078\n",
            "[115] [it=90000] Train Loss: 0.815, Test Loss: 1.087\n",
            "[116] [it=91000] Train Loss: 0.824, Test Loss: 1.095\n",
            "[117] [it=92000] Train Loss: 0.813, Test Loss: 1.082\n",
            "[118] [it=93000] Train Loss: 0.817, Test Loss: 1.081\n",
            "[120] [it=94000] Train Loss: 0.806, Test Loss: 1.082\n",
            "[121] [it=95000] Train Loss: 0.808, Test Loss: 1.089\n",
            "[122] [it=96000] Train Loss: 0.809, Test Loss: 1.072\n",
            "[124] [it=97000] Train Loss: 0.807, Test Loss: 1.087\n",
            "[125] [it=98000] Train Loss: 0.802, Test Loss: 1.091\n",
            "[126] [it=99000] Train Loss: 0.804, Test Loss: 1.083\n",
            "[127] [it=100000] Train Loss: 0.806, Test Loss: 1.085\n",
            "[129] [it=101000] Train Loss: 0.803, Test Loss: 1.096\n",
            "[130] [it=102000] Train Loss: 0.798, Test Loss: 1.089\n",
            "[131] [it=103000] Train Loss: 0.790, Test Loss: 1.096\n",
            "[132] [it=104000] Train Loss: 0.799, Test Loss: 1.102\n",
            "[134] [it=105000] Train Loss: 0.799, Test Loss: 1.079\n",
            "[135] [it=106000] Train Loss: 0.794, Test Loss: 1.096\n",
            "[136] [it=107000] Train Loss: 0.798, Test Loss: 1.102\n",
            "[138] [it=108000] Train Loss: 0.792, Test Loss: 1.096\n",
            "[139] [it=109000] Train Loss: 0.786, Test Loss: 1.104\n",
            "[140] [it=110000] Train Loss: 0.790, Test Loss: 1.102\n",
            "[141] [it=111000] Train Loss: 0.785, Test Loss: 1.105\n",
            "[143] [it=112000] Train Loss: 0.780, Test Loss: 1.102\n",
            "[144] [it=113000] Train Loss: 0.787, Test Loss: 1.095\n",
            "[145] [it=114000] Train Loss: 0.785, Test Loss: 1.099\n",
            "[147] [it=115000] Train Loss: 0.778, Test Loss: 1.099\n",
            "[148] [it=116000] Train Loss: 0.781, Test Loss: 1.087\n",
            "[149] [it=117000] Train Loss: 0.781, Test Loss: 1.081\n",
            "Done training.\n"
          ]
        }
      ],
      "source": [
        "training_loss_d, testing_loss_d = [], []\n",
        "running_loss = []\n",
        "i = 0\n",
        "for epoch in range(150): # 10 epochs\n",
        "  for data in train_loader:\n",
        "    inputs, labels = data\n",
        "    # get the data to GPU (if available)\n",
        "    inputs, labels = inputs.to(device), labels.to(device)\n",
        "\n",
        "    optimizer.zero_grad()\n",
        "    # forward pass\n",
        "    outputs = net_dropout(inputs)\n",
        "    # backward pass\n",
        "    loss = criterion(outputs, labels)\n",
        "    loss.backward()\n",
        "    # update gradients\n",
        "    optimizer.step()\n",
        "    running_loss.append(loss.item())\n",
        "    i += 1\n",
        "    if i % 1000 == 0:\n",
        "      avg_train_loss = sum(running_loss) / len(running_loss)\n",
        "      avg_test_loss = get_test_loss(net_dropout, criterion, test_loader)\n",
        "      # clear the list\n",
        "      running_loss.clear()\n",
        "      # for logging & plotting later\n",
        "      training_loss_d.append(avg_train_loss)\n",
        "      testing_loss_d.append(avg_test_loss)\n",
        "      print(f\"[{epoch:2d}] [it={i:5d}] Train Loss: {avg_train_loss:.3f}, Test Loss: {avg_test_loss:.3f}\")\n",
        "\n",
        "  \n",
        "print(\"Done training.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        },
        "id": "kzLI4l9mHgiD",
        "outputId": "adee9a9d-ecfa-4a54-d745-ef76281fc822"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEICAYAAAAKrmAuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3xV5f3A8c83g2wSSFhJCAl7hL1BpoqKA7XaugVXxVZtq9a2v9ZiraN1Vq21VQtO3OLCgWUje++wIWFkkb2T5/fHcwIhZAG5uUnu9/165ZV7z/yec8853/M85znniDEGpZRSqrHxcncASimlVFU0QSmllGqUNEEppZRqlDRBKaWUapQ0QSmllGqUNEEppZRqlM4pQYnIDBF5p76CUa4jIgtF5A53xwEgItNF5JiI5IhIeB2GnyoiSxsiNtW4tpVyIrJVRMbX0N9tMYvIfhG5wB3zbu5qTFDOAaT8r0xE8it8v7G+gxGRYSIyV0QyRCRdRFaJyLT6nk9DEJFYETEi4uPuWBoTEfEFngMmGWOCjTFplfq7dL05J1VGRO6v1P1+p/sM5/t45/srlYZbKiJTnc+nJE4ROU9EfhSRTGf7XSYiQ0XkDxX2mwIRKa3wfWsVMdbbOhCRWSLy13OdTg3Tb5CDszGmjzFmoTPPZnliLCItRCRVRILPcLxZIlIkItnO3xYReVJEQl0V67k4k22mxgTlHECCjTHBwEHg8grd3q2PYMuJyEhgPrAI6AqEA9OBS6oZvskf+JvDMpyFdoA/cNqBuQElALdU6nar072iXOBmEYmtbYIi0hL4CngJaA1EAY8ChcaYJyrsR3cDyyvsR33OaUlUczIW2GCMyTmLcf9ujAkB2gDTgBHAMhEJqmrgpnLsqY9rUC1E5C0nc28VkSHlPUQkUkQ+EZEUEdknIvfVMJ2ngTeNMX8zxqQaa60x5qfOtMaLSKKIPCwiR4GZIuInIi+IyGHn7wUR8XOGjxCRryqUxpaIiJfT72ERSXJi3iki51cXlIiMcM6KM0RkY8VqBqda4THnTDlbRL4XkQin92Lnf4ZzpjzSOeNeJiLPi0gaMENEQp31lyIiB0TkjxXiLB/+ZeesfEd5rCJyrYisrRTrb0Tk89p+MBHxcuZzQESSnfmHOv38ReQdEUlzlnm1iLSrEM9eZ1n3STWl6Op+FxHpDuyssF7mVzH6aeutwnSfEZHjzrwvqdA9VETeEJEjzu/6VxHxrmEVrAYCRaSPM34fbNJcXWm4DGAW8OcaplWuO4AxZrYxptQYk2+M+d4Ys6kO41ZW5ToQkdtEZLuzDr4TkU5Od3G2qWQRyRKRzSISLyJ3ATcCv3Wm82VVMxORC51tK1NEXgakQr8uIjLf2R5SReRdEQlz+r0NxABfOtP/rdP9IxE56kxvcfl6rmK+E0Rkc4Xv80RkdYXvS0TkSufzfhG5QEQuBv4A/MyZ58YKk+xUzb5Yeb6txB4bUpx1+ZWIRFfoX9N+jYjc7Ow7aSLyf1XNwxkuztmHyvfn10QkuUL/t0XkVxVGmQzMdfpFisgXYo9du0XkzurmU5ExpsAYsxq4AnuSP82ZXr0de2qLTyqV2sU5dpcvM1VsMzUtUJ3+gP3ABZW6zQAKnBXrDTwJrHD6eQFrgUeAFkBnYC9wURXTDgRKgQk1zH88UAL8DfADAoC/ACuAttgzhx+Bx5zhnwReBXydvzHYHa8HcAiIdIaLBbpUM88oIM1ZPi/gQud7G6f/QmAP9uAU4Hx/qsJ0DeBTYXpTnWW4F/BxxnkL+BwIccZJAG6vNPyvnWX4GZCJPUP3A9KBXhWmvx74STXLshC4w/l8G7Db+U2CgU+Bt51+Pwe+dH4Tb2Aw0BIIArKAHs5wHYA+1cyrpt/ltPVSadzq1lsxcKcT03TgMCBO/8+AfzsxtgVWAT+vZvozgHewB7m/Od3+Dvze6T6jwvaWCLSvtNxLgakV4lrqfG6J3TbexJb6W1Uz/xPj1LCtV7UOpji/WS9n2/kj8KPT7yLsvhaG3cZ7AR2cfrOAv9YwrwggG7gGu439GrvNlW8rXbHbvZ/zWy4GXqjluHAbdnv2A17AlgqqmncA9vgR4cz7GJDkjBsA5APhledT/htWsX1XuS9WMd9w4CfYbTwE+AiYU5dpAb2BHGxpxw9bXV1SeR1UmNZBYLDzeSf2GNirQr+BFYbdwcntbDHwCvbEaQCQAkysZh5V/sbYY8sH9X3sqS2+yvHg7Es1bTPV/dVHCWqpMWauMaYUeBvo73Qfij2Q/8UYU2SM2Qu8BlxXxTRaYRPAkVrmVQb82RhTaIzJx54d/sUYk2yMScFWqdzsDFuMPYh2MsYUG2OWGLt2SrEbVm8R8TXG7DfG7KlmfjcBc53lKzPGzAPWYBNWuZnGmAQnng+xP1ZNDhtjXjLGlABF2PXxe2NMtjFmP/BshWUASMYeEIqNMR9gN/JLjTGFwAdOjOWlgFhsNVNtbgSeM8bsNbY64ffAdWKL/cXYHbirsSWBtcaYLGe8MiBeRAKMMUeMMdVV09X0u5ytA8aY15zt7E3sb9tObOluMvArY0yuMSYZeJ6qt7OK3gGuF3tN7Drn+2mMMUexJzp/qWlizjo6D5tYXgNSnDPMdnVewprdDTxpjNnubDtPAAOcUlQx9iDTE5u0txtjatuXyk0GthpjPjbGFGMTytEKy7XbGDPP2edSsAfkcTVN0BjzX2d7LsQmk/5SxfUQZ59ZjT3YDwY2AsuA0dgqql2m0jXKWtRpXzTGpBljPjHG5BljsoHHq1im6qZ1DfCVMWaxs3x/wu4X1VkEjBOR9s73j53vcdiTmo1gS6rYE5KdItLRWQcPG1si2gC8zunV0rU5jD2ZPfG9Po499RhfreojQR2t8DkP8HcOdJ2ASKeImyEiGdiz1qp22OPYH7lDLfNKMcYUVPgeCRyo8P2A0w1sleFu4Hux1VK/A7vDAb/C7jjJIvK+iETCaY1CYpxluLbSMpxXKc7Ky1/bBc5DFT6XnzlWXoaoCt+TnMRa1TK+CdwgIoLdsD50dpraVLXefLC/zdvAd8D7Yqvn/u4k8lzsWdTdwBER+VpEep7B9COrGbauKh4085yPwdjfyNeJqfw3+je2JFUtY8xB7PbxBPZAeKiGwf8GXCQi/WsYBicxTDXGRAPx2GV+oebFqrNOwD8qLGM6trQUZYyZD7wM/BO7Tf9H7DWxuoikwjbpbGsnvotIO2cfSRKRLGwir7LqzBneW0SeEpE9zvD7nV7VjbMIe4Y91vm8EJssxjnfz0Sd9kURCRSRfzvVWlnY0kCYnFotXN20Kq+vXGzJuToVl28xpy7fEmNMeXKbDHxTYR7pTvIsV/m4UBdR2O2kXH0de+orvlq58j6oQ8A+Y0xYhb8QY8zkygM6B5zl2GJ3TSo/ev0wdsctF+N0wzkreMAY0xlbH/ub8jpUY8x7xpjznHEN9gB0SqMQ5wB2CFv1VXEZgowxT9Vh+at7THzF7qnYs9/Ky5BU4XuUk4CqWsYV2DOhMcAN2ORSF1WttxLgmHO29KgxpjcwCrgM58zIGPOdMeZCbILegS0p1HX6h+sY25k+Xv8QUAhEVPiNWpq6NT54C3jA+V99QPYs/gXgsboGZYzZga3qiK/rOBVHr6LbIWy1ZcVtMcAY86MzvxeNMYOxVVDdgYdqmFZFR4CO5V+cba1jhf5PONPoa4xpiS2xV9weK0//Bmx15AVAKLZUT6VxKqqcoBZRe4I611cwPICt6h/uLNPYWmKsqPL6CsTWOFRnEXb/HO98XootfVRevhPXn3BKPiISUqF/5eNCjcS2BLwAWFKhc30de2qLLxdbfVquPaeq8+/nygS1CsgW2yAhwDmziheRodUM/1tgqog8JM69MSLSX0Ter2Ees4E/ikgb5yLmIzhVNSJymYh0dVZwJrZqr0xEeojIRLGNKQqw9dzVFdHfAS4XkYuc+P2dC37R1QxfUYoz3c7VDeBUV30IPC4iIU51zW84tbqpLXCfiPiKyLXY6wtzK/R/C3v2XGyMqeu9QrOBX4u9iBuMPQh9YIwpEXvhuq9zNpmF3YjLnDPpKWJbBRVi6+GrW2/V/i51UOt6q8ipyvoeeFZEWoptANJFRGqshnJ8AEzC/ga1eQ6bsHtV1VNEeorIA+XbhlMNcj32WtyZqmodvAr8Xk427Ah1tgfENmUf7lRX5mK36/Lf5hg1r8uvgT4icrVT83Efpx5QQrC/daaIRHEy8ZWrPP0Q7PaRhj1IPVHLsv6ITRbDgFVOtXEnYDgnG4tUdgyIFeeC/lkIwe73GSLSmro1gin3MXCZ2FsKWmCrfquNwxizy5nXTcAipyr4GPZkfBGcSHLDgAXOOIew6+VJ55jTD7idOuxDYhsjDQbmYGumZlYT11kfe+oQ3wZgsoi0Flu1WbEhCNS+TZ7gsgTlrIDLsHW3+7AZ+3XsWVVVw/8ITHT+9opIOvAfTj0YV/ZX7DWhTcBmYJ3TDaAb8AN251oOvGKMWYC9/vSUE89R7I/w+2piOoQ9G/wD9qBxCLuD1rrenFLh49imnhkiMqKaQe/FHlT2Ys+u3gP+W6H/SmdZUp3pXVOpXv5t7Fn6mdwX8l9nvMXY36bAiQPsweljbHLajt2J3sYu82+wZ0/p2DPA6dVMv6bfpUZnsN4qugXbEGcbdqf8mNqrizG2pd0PznWG2obNwjamaF3NINnYg+pKEcnFJqYt2LP1M1LVOjDGfIYt6b/vVEtt4eQtGC2xpdnj2KqWNGwVN8Ab2OutGSIyp4p5pQLXYveJNOy2tqzCII8Cg7AneV9jG9RU9CT2ZCRDRB7EnjAdwJ5Nb6OWBO1Uka3DXgcrcjovx15zTK5mtI+c/2kisq6m6VfjBWwjgVQnvm/rOqKTQH+B3U+PYNd5Yi2jLQLSKlQjL8KW1spjn4i99aDi5YvrsaXPw9hGQH82xvxQwzx+KyLZ2N/wLWyjmVHO+q3OuRx7aorvbey1tf3Yk8cPKs238jZTrfJWUKoREntD6B1OdWR1wwRgL2YOcs7WlFJNiNibwbcYY16pdeAGUpdjT0NoEjdrqRpNB1ZrclKqydqAvbVDVaIJqgkTkf3YqoIr3RyKUuosGWP+4+4YGiut4lNKKdUo6es2lFJKNUpNsoovIiLCxMbGujsMpZRqMtauXZtqjGnj7jjORJNMULGxsaxZs8bdYSilVJMhIgdqH6px0So+pZRSjZImKKWUUo2SJiillFKNUpO8BlWV4uJiEhMTKSgoqH1g5TL+/v5ER0fj6+vr7lCUUk1cs0lQiYmJhISEEBsby6kP4FUNxRhDWloaiYmJxMXFuTscpVQT12yq+AoKCggPD9fk5EYiQnh4uJZilVL1otkkKECTUyOgv4FSqr40myo+pZRqbkrLDB+vPURGXjEtA3xp6e/LRX3a4ePdrMoW1dIE1UgFBweTk5PjlnnPmjWLSZMmERl5rm9pV0qdi3dWHODPX2w9pdueJ057KXmzpQmqCSkpKcHHx/U/2axZs4iPj9cEpZQbHc7I5+/f7mBs9za8cuMgsguKySkowdvLc6rRPaOc2AD2799Pr169uPPOO+nTpw+TJk0iP9++qHXDhg2MGDGCfv36cdVVV3H8+PHTxt+3bx8jR46kb9++/PGPfzzRfeHChYwZM4YrrriC3r17U1BQwLRp0+jbty8DBw5kwYIFgE0qU6ZMYfz48XTr1o1HH330xDSee+454uPjiY+P54UXXjgRb3x8/IlhnnnmGWbMmMHHH3/MmjVruPHGGxkwYMCJZVBKNRxjDI98voUyA49fGU+wnw8dQgPo1i7E3aE1qGZZgnr0y61sO5xVr9PsHdmSP1/ep8Zhdu3axezZs3nttdf46U9/yieffMJNN93ELbfcwksvvcS4ceN45JFHePTRR08kinL3338/06dP55ZbbuGf//znKf3WrVvHli1biIuL49lnn0VE2Lx5Mzt27GDSpEkkJCQAsGrVKrZs2UJgYCBDhw7l0ksvRUSYOXMmK1euxBjD8OHDGTduHK1atapyGa655hpefvllnnnmGYYMGXIOa0wpdTYy84v5dF0iP2xP5v8m96Jj60B3h+Q2zTJBuUtcXBwDBgwAYPDgwezfv5/MzEwyMjIYN24cALfeeivXXnvtaeMuW7aMTz75BICbb76Zhx9++ES/YcOGnbivaOnSpdx7770A9OzZk06dOp1IUBdeeCHh4eEAXH311SxduhQR4aqrriIoKOhE9yVLlnDFFVe4YhUopc7Q4oQUVu9PZ19qLruO5ZCQnI0xMKBjGNNGx7o7PLdqlgmqtpKOq/j5+Z347O3tfcbVY9U10S5PLmc6fk1Nvn18fCgrKzvxXe9dUqrhfb3pCL94bx1eAtGtAomLCOLSfh0YEtuKQTGtPKa1XnU8e+kbQGhoKK1atWLJkiUAvP322ydKUxWNHj2a999/H4B333232umNGTPmRP+EhAQOHjxIjx49AJg3bx7p6enk5+czZ84cRo8ezZgxY5gzZw55eXnk5uby2WefMWbMGNq1a0dycjJpaWkUFhby1VdfnZhHSEgI2dnZ9bYOlFKn23Y4iwc/2sigmDC2/eViFv92Am/eNoz7zu/GqC4R+Pt6uztEt2uWJajG5s033+Tuu+8mLy+Pzp07M3PmzNOG+cc//sENN9zA3/72N6ZMmVLttO655x6mT59O37598fHxYdasWSdKbsOGDeMnP/kJiYmJ3HTTTSeuIU2dOpVhw4YBcMcddzBw4EAAHnnkEYYNG0ZUVBQ9e/Y8MY+pU6dy9913ExAQwPLlywkICKi3daGUp3l9yV5iw4O4oHe7E93Sc4u46+01tAzw4dWbBmsyqoYYY9wdwxkbMmSIqfzCwu3bt9OrVy83ReR+s2bNYs2aNbz88svuDsXjfwulypWUlhE/4zv6RYfx4c9Hnuj+xzmb+XB1Ih/ePZIBHcMaJBYRWWuMaVItn7SKTymlXGRvai4FxWVsScqktOxkYWDVvnRGdw1vsOTUVGmCaiamTp3aKEpPSqmTtiRlApBXVMqeFPtkmOyCYnYl5zAwpupbPdRJmqCUUspFtiSdvB9z46EMADYlZp5oRq5qpglKKaVcZEtSJgM6hhHs58PGRJug1h+0T5LprwmqVpqglFLKBcrKDFsPZ9IvOpS+UaFsSrTVfesPZtC1bTChAfrW6dpoglJKKRfYn5ZLblEp8ZGh9OsYyvYjWRSWlLLhUIZW79WRJqgGNHnyZDIyMsjIyOCVV1450X3hwoVcdtllZzStyg97bWgvvPACeXl5bpu/Uo3dFud5oH2iWtI/OoziUsO8bcdIyy1iYIwmqLrQBNWA5s6dS1hY2GkJqj6VlJS4ZLqVaYJSqmZbkjJp4e1F93Yh9IsOBeDNH/cD2kCirjRB1ZOnn36aF198EYBf//rXTJw4EYD58+dz4403AhAbG0tqaiq/+93v2LNnDwMGDOChhx4CICcnh2uuuYaePXty4403UtUN1GvXrqV///7079//lCeez5o1iyuuuIKJEydy/vnnk56ezpVXXkm/fv0YMWIEmzZtAmDGjBncfPPNjBw5km7duvHaa68B9tH+Dz30EPHx8fTt25cPPvgAOL1k98tf/pJZs2bx4osvcvjwYSZMmMCECRPqe1Uq1SxsScqkZ4cQfL29iAoLIDyoBav3HyfA15seHvbajLPVPB919M3v4Ojm+p1m+75wyVPV9h4zZgzPPvss9913H2vWrKGwsJDi4mKWLFnC2LFjTxn2qaeeYsuWLWzYsAGwiWD9+vVs3bqVyMhIRo8ezbJlyzjvvPNOGW/atGm8/PLLjB079kRiK7du3To2bdpE69atuffeexk4cCBz5sxh/vz53HLLLSfmtWnTJlasWEFubi4DBw7k0ksvZfny5WzYsIGNGzeSmprK0KFDT4u5ovvuu4/nnnuOBQsWEBERcUarUSlPYIxhS1Iml/azL/0UEfpFh7JgZwp9o0M9/iGwdaVrqZ4MHjyYtWvXkpWVhZ+fHyNHjmTNmjUsWbKEMWPG1Dr+sGHDiI6OxsvLiwEDBrB///5T+pdfuypPHDfffPMp/S+88EJat24N2FdylPefOHEiaWlpZGXZ+vApU6YQEBBAREQEEyZMYNWqVSxdupTrr78eb29v2rVrx7hx41i9evW5rhKlPFbi8XyyCkqIj2p5olu/aFutp9ef6q55lqBqKOm4iq+vL3FxccyaNYtRo0bRr18/FixYwO7du+v0XLrKr+o402tJ+koOpRqHsjLDwoQUAOIjQ090L09Mg/QJEnWmJah6NGbMGJ555hnGjh3LmDFjePXVVxk4cOBpSeBsXmcRFhZGWFgYS5cuBer+So6FCxcSERFBy5b2TO7zzz+noKCAtLQ0Fi5cyNChQxkzZgwffPABpaWlpKSksHjxYoYNG0anTp3Ytm0bhYWFZGRk8L///e+clkGp5qyguJQHPtzIoL/O409zthDi50OP9ievNY3t1oZ/3zyYC3q1q2EqqqLmWYJykzFjxvD4448zcuRIgoKC8Pf3r7J6Lzw8nNGjRxMfH88ll1zCpZdeWqfpz5w5k9tuuw0RYdKkSdUON2PGDG677Tb69etHYGAgb7755ol+/fr1Y8KECaSmpvKnP/2JyMhIrrrqKpYvX07//v0REf7+97/Tvn17AH76058SHx9PXFzcidd0ANx1111cfPHFREZGsmDBgrquIqWarVcX7eGTdYlcPTCKMd0jGN311Hc6eXkJF/Vp78YImx593YYHmTFjBsHBwTz44IMunY/+FsrTHEzL44LnFzGpdztevmGQu8Opkr5uQymlPNCjX27Fx0v446W93R1Ks6JVfB5kxowZ7g5BqWbDGENabhHzth3jfzuS+cPknrQP9Xd3WM1Ks0pQxpgaW6Up12uKVcZKnamFO5N58KONpOYUAdCzfQjTRse5Oarmx6UJSkT8gcWAnzOvj40xf640jB/wFjAYSAN+ZozZf6bz8vf3Jy0tjfDwcE1SbmKMIS0tDX9/PYtUzdf+1Fzunb2eDqH+3DO+K13bBjOoUyt89ebbeufqElQhMNEYkyMivsBSEfnGGLOiwjC3A8eNMV1F5Drgb8DPznRG0dHRJCYmkpKSUj+Rq7Pi7+9PdHS0u8NQyiXyikq4+521eInwxq1D6dg60N0hNWsuTVDG1vfkOF99nb/KdUBTgBnO54+Bl0VEzBnWFZXfKKuUUq7yx8+2sPNYNrOmDdPk1ABcXiYVEW8R2QAkA/OMMSsrDRIFHAIwxpQAmUB4FdO5S0TWiMgaLSUppRpaUkY+n65P4udjuzCuext3h+MRXJ6gjDGlxpgBQDQwTETO6iVGxpj/GGOGGGOGtGmjG4dSqmHN23oUgJ8O0SrshtJgV/WMMRnAAuDiSr2SgI4AIuIDhGIbSyilVKMxb/sxurYNpnObYHeH4jFcmqBEpI2IhDmfA4ALgR2VBvsCuNX5fA0w/0yvPymllCtl5hWzYm86F/bW5+g1JFe34usAvCki3thk+KEx5isR+QuwxhjzBfAG8LaI7AbSgetcHJNSSp2RBTuTKS0zTNIE1aBc3YpvEzCwiu6PVPhcAFzryjiUUupM5ReVEtDCPuz1+21HaRviR/9ofZdTQ9I7y5RSqpItSZnEz/iO//tsM1kFxSzcmcIFvdvh5aUPAWhIzepRR0opVR++23qU0jLDuysPMn9HMnlFpXr9yQ20BKWUUpUsSkhhcKdW/PvmweQUlBDi58OoLqfdnqlcTEtQSilVQWpOIZsSM3ngwu5c1Kc9fX8dSmZ+MX4+3rWPrOqVlqCUUh7t47WJjH5qPpl5xQAs2WWfVDOuh30gQGRYAL06tHRbfJ5ME5RSyqPNWZ9EUkY+byzdC8CinSmEB7UgPjLUzZEpTVBKKY+VW1jCqn3p+HgJ/122n7ScQhbvSmVs9zbaYq8R0ASllPJYK/amUVRaxh8m9yK3qIRff7iR9NwifRhsI6GNJJRSHmvhzhQCW3hz44gY1h/K4MuNhxGBMd0i3B2aQktQSikPZYxhYUIyo7qE4+fjzf3nd0UE+kWFEh7s5+7wFFqCUkp5qL2puRxKz+eusV0A6No2hMemxBOjLyJsNDRBKaU8xtoDxzmWVcAl8e1ZuNM2Jx9f4XrTTSM6uSs0VQVNUEopj/HnL7awJSmLsd3bkJVfTOc2Qfrq9kZME5RSyiMUFJey40g2AzqGsWZ/OnlFpdw2Os7dYakaaIJSSnmEbUeyKCkz3D2uC30iW/L6kr1MHRXr7rBUDTRBKaU8wqZDGQD07xhKh9AAHp0S7+aIVG20mblSyiNsSsykTYgf7Vv6uzsUVUeaoJRSHmFDYgb9o0MR0UcYNRWaoJRSzV5WQTF7U3L1le1NjCYopVSztyUxE4B+HTVBNSWaoJRSzd7G8gQVpa/QaEo0QSmlmr2NhzKIaR1Iq6AW7g5FnQFNUEqpZulIZj55RSUAbErMoL9W7zU5eh+UUqrZ2XY4i6teWUaQnw83Do/hcGYBt0Vr9V5TowlKKdWsZBcU84v31hEa4EvPDi15af5uAPppC74mRxOUUqrJM8YgIhhj+N2nmzmYnsd7dwxneOdw1h08zup96Qzu1MrdYaozpAlKKdWkJRzL5sp/LsPX24tWgb7sT8vj4Yt7MrxzOACDYloxKEaTU1OkCUop1aTNXLaPMmO4on8kydkFXNK3Az8f29ndYal6oAlKKdVkZeYXM2f9Yab0j+KxK/Xhr82NNjNXSjVZn6xNJL+4lJtH6ptwmyNNUEqpJqmszPDOigMMjAkjXp8Q0SxpglJKNUnL9qSyNzWXW7T01GxpglJKNUlvLT9A66AWTO7bwd2hKBdxWYISkY4iskBEtonIVhG5v4phxotIpohscP4ecVU8SqnmY19qLj9sP8b1wzri5+Pt7nCUi7iyFV8J8IAxZp2IhABrRWSeMWZbpeGWGGMuc2EcSqlm5o2le/H18uLWUbHuDkW5kMtKUMaYI8aYdc7nbGA7EOWq+SmlPENaTiEfrUnkqoFRtA3R17c3Zw1yDUpEYoGBwMoqeo8UkY0i8o2I9KlhGneJyBoRWZOSkuKiSDgOnxQAACAASURBVJVSjd1byw9QWFLGnWPj3B2KcjGXJygRCQY+AX5ljMmq1Hsd0MkY0x94CZhT3XSMMf8xxgwxxgxp06aN6wJWSjVa+UWlvL3iABf0akvXtiHuDke5mEsTlIj4YpPTu8aYTyv3N8ZkGWNynM9zAV8RiXBlTEqppmlLUia/eG8d6blF3DlGH2XkCVzWSEJEBHgD2G6Mea6aYdoDx4wxRkSGYRNmmqtiUko1PRl5Rfzmw43M35FMiL8PD13Ug2Fxrd0dlmoArmzFNxq4GdgsIhucbn8AYgCMMa8C1wDTRaQEyAeuM8YYF8aklGpC0nIKuemNVexJzuGhi3pw88hOtPT3dXdYqoG4LEEZY5YCUsswLwMvuyoGpVTTlZxdwE2vr+RAWh6v3zqEsd312rOn0aeZK6Uapftnb+BQej4zpw1lVBe9NO2J9FFHSqlGZ+2B4yzfm8YDk7prcvJgmqCUUo3OvxbuISzQl+uHxbg7FOVGmqCUUo1KwrFsfth+jKmjYgny06sQnkwTlFKqUXl14R4CfL25dWSsu0NRbqYJSinVaKw7eJzPNx7m+mExtApq4e5wlJtp+Vkp5XYH0nJ55vsEvtx4mIjgFvqcPQVoglJKuVl6bhGXv7SU4lLDvRO7cufYznozrgI0QSml3Ozl+bvJKSxh7v1j6Nm+pbvDUY2IXoNSSrnNofQ83llxgGsHd9TkpE6jCUop5TbPz0tABH51YTd3h6IaIY9KUEt3pbLhUIa7w1BKAdsOZ/HZhiSmjo6lQ2iAu8NRjZBnJaiPXmDFt++5OwylPF5GXhG/+mA9Lf19uWdcV3eHoxopj0pQN5V+xqD0r90dhlIeLa+ohNtmrWZ/ah7/unEQoYHaYk9VzaMSVJZvW1oWJbs7DKU8VklpGdPfWceGQxm8eP0ARnXVB8Gq6nlUgsoLaE94aYq7w1DKY83bdoxFCSk8OiWei+M7uDsc1ch5VIIqCupAuMmgrLjQ3aEo5ZHmbEiiTYgfN+hTylUdeFSCKg2JwksMOamH3B2KUh4nM6+YBTtSuLxfJN5eNb5sWynAwxKUd1g0AFnHDrg5EqU8zzdbjlBUWsaVAyPdHYpqIjwqQfmFdwSgMO2gmyNRqvlauDOZI5n5p3WfsyGJzhFB9I0KdUNUqinyqAQVFBELQMlxreJTyhX2peYydeZqJj6ziH/8sIv8olIAjmTms3JfOlMGRCGi1XuqbjzqYbGtWrcmywRCVpK7Q1GqWfpxTyoAQ+Na8/wPCby78gCT+3agqLQMY2DKAK3eU3XnWQkqsAV7TTi+OYfdHYpSzdLyPWm0a+nHm9OGsmpfOq8v3cd7qw5SVFLGgI5hxEYEuTtE1YR4VIJq4eNFilcEXfKPujsUpZodYwwr9qZzXtdwRIThncMZ3jmcnMISliSk0L19iLtDVE2MRyUogAyftoQUrXB3GEo1O7uTc0jNKWRkl/BTugf7+XBJX70pV505j2okAZDj146Q0kwoPr2VkVLq7C3fmwbAyM76+CJVPzwuQRUEtrcfsvQ6lFL1acXeNKLCAujYWl+doeqHxyWokpAo+yEz0b2BKNWMlJXZ608jOodrM3JVbzwuQXmF2gRVpglKqXqTkJxNem4RIzq3dncoqhnxuEYSvq3s444KUg8S6OZYlGrqjmTmYwzM32FfY1O5gYRS58LjElRYaChpJgQffZqEUufk8w1J3P/+hhPfY1oHEt1KT/tU/fG4BBUR1IIjJpyoDK3iU+psFZeW8ez3CfRoF8K00bEUlZbRPzrM3WGpZsbjElTr4BYcMuHE6NMklDprn6xN5GB6Hm/cOoTze7VzdziqmXJpIwkR6SgiC0Rkm4hsFZH7qxhGRORFEdktIptEZJArYwoP8uOwaY1/nj5NQqmzUVhSykvzdzOgYxgTe7Z1dziqGXN1K74S4AFjTG9gBPALEeldaZhLgG7O313Av1wZUKtAX46YcFqUZENhtitnpVSz9OHqQyRl5PObC7trk3LlUi6t4jPGHAGOOJ+zRWQ7EAVsqzDYFOAtY4wBVohImIh0cMatdz7eXmS2aAsGyEyCtj1dMRulmpXM/GI+XH2IH/eksnxvGkNjWzGmmz4xQrlWg12DEpFYYCCwslKvKKBik7pEp9spCUpE7sKWsIiJiTmnWPL8O0A+kHFQE5RSdfCnOVv4YuNhOkcEcfWgaKaP66KlJ+VyDZKgRCQY+AT4lTEm62ymYYz5D/AfgCFDhphziSe7ZReboFJ2QPdJ5zIppZq97IJivtt6lJtHdOKxK+PdHY7yIC5/koSI+GKT07vGmE+rGCQJ6Fjhe7TTzWX8QiJIldaQvN2Vs1GqSXpnxQG+2XyyAuPbLUcpLCnjyoFRboxKeSKXlqDE1gG8AWw3xjxXzWBfAL8UkfeB4UCmq64/lQsPbsEuE01E8rbaB1bKgxzNLGDGF1vx9/VmaFxrIoL9mLMhiZjWgQyK0fucVMNydQlqNHAzMFFENjh/k0XkbhG52xlmLrAX2A28Btzj4phoHeTH1pIoTMpOKCt19eyUajLeXL6fMmMoKC7l+XkJHMsq4Mc9aVw5MEqvOakG5+pWfEuBGrdqp/XeL1wZR2URwS3YbDoiJflwfD+Ed2nI2SvVKOUVlfDeyoNc1Kc97Vr689by/RSVlGEMXDkg0t3hKQ/kcU8zB2gd1IKEMvvQWL0OpZT1ydpEMvOLuf28OO4/vxvBfj58tDaR/h3D6Nwm2N3hKQ/kkQkqPMiPXUYTlFLlysoM/122n/4dwxjcqRWtglpw3/ndAC09KffxuGfxAXRsHUAe/uQERBGsDSWUYv6OZPal5vLi9QNPXGu6dVQsoQG+XN5fE5RyD48sQXUIDaCFtxdH/OK0BKUU8PrSvXQI9eeS+PYnuvl6e3HtkI74+3q7MTLlyTwyQXl7CTHhgeymI6TtgpIid4eklNtsScpkxd50po2OxdfbIw8JqpHy2K0xNjyQjUWRUFYCabvdHY5SbvPakr0E+/lw3bBze4SYUvXNYxNUp/Agfsx2XhWg16GUhzqckc9Xm47ws6Edaenv6+5wlDqFxyao2PBAdhS3x4i3XodSHmvWj/sBmDY61q1xKFUVz01QEUEU4Ut+S20ooZq/zLxiXlu8l5zCkhPdEo/nMXvlQS6Jb090q0A3RqdU1Tw3QYUHAZAa2BmObXFzNEq51jPf7+TxuduZNnMVOYUlZOQVMXXmakTg1xd2d3d4SlXJI++DAugQ6o+vt7DbtycxR76H7KMQ0r72EZVqYg6l5/H+6oMMjAlj3cEMps1chSAcTMvjrduH0UWfEqEaKY8tQfl4e9GxVSCrjfPCwgM/ujcgpVzkxf/tQkR45cZBvHjdQNYdzGDV/nSe+1l/RnQOd3d4SlXLY0tQYK9DLcmI4mHfIJug4q92d0hK1as9KTl8si6RaaPj6BAawKX9AggN8CWnsISL47XGQDVuHp2gOoUHsmJvGqbbcERLUKoZen5eAv6+3kwff/KJ/ed1i3BjRErVncdW8YFtKJFXVEpu+6H2Xqi8dHeHpFS9WbE3ja82HeH28+KICPZzdzhKnTGPTlCdwm3T2kMhAwEDh1a6NyCl6klhSSl/+Gwz0a0CuGd8V3eHo9RZ8egEFRdhm5pvk27g3QIOLHNzRErVj1cX7mVvSi5/vTKegBb6sFfVNHl0gooKC8DHS9iXUQpRQ7Qln2oW9qbk8M8Fu7m8fyTje7R1dzhKnTWPTlA+3l5EtwpgX1oudBoFhzdAYY67w1LqrGUXFHPPu+vw8/XiT5f1cnc4Sp0Tj05QYB8ae6A8QZlSSFzl7pCUOivFpWX84r317ErO4Z83DKJtiL+7Q1LqnHh8gurRPoSEoznktRsEXj6w6wd3h6TUGTPG8Kc5W1ickMITV8Uztnsbd4ek1Dnz+AQ1tlsbikrLWJFUBL0uhw3vaDWfanK+2HiY91cf4hcTuvCzofpeJ9U8eHyCGhrXigBfbxbuTIHh06EgEzbOdndYStVZVkExf/16O/2iQ/nNhT3cHY5S9cbjE5SfjzejuoSzKCEFOg6DyEGw8t9QVubu0JSqk2e/20laTiGPX9kXby9xdzhK1RuPT1AA43q04UBaHvvS8mDEdEjbBXv+5+6wlKrVpsQM3lpxgJtHdKJvdKi7w1GqXnn0s/jKje/eFtjKop3JxA2/Er7/E6x4Bbpd6O7QlDrNvtRcnpuXwO7kHPam5BAR7McDF2nVnmp+tAQFxIQH0jkiiIUJKeDTAobeAXvmw359soRqXErLDL96fz0LdiTTvqUfN43oxKxpQ2np7+vu0JSqd1qCcozt3obZqw5SUFyK/4jpsOFd+OzncPdSCAhzd3hKATB71UE2Jmbyj+sGMGVAlLvDUcqltATlGN+jDYUlZazclw5+wfCT1yHrMMx90N2hKQVAak4hf/92B6O6hHNF/0h3h6OUy2mCcozoHI6/rxc/bDtmO0QPgXEPw+aPYNNH7g1Oeay0nEIW7Ejm+61H+b/PNpNfXMpfpsQjoq31VPOnVXwOf19vzu/Vjrmbj/DI5b3x9faCMQ/A7nnw/f9Br8vAN8DdYSoPsv7gce58aw2pOUUnut1/fje6tg12Y1RKNRxNUBVM6R/J15uOsGx3qn0KtLcPXPAozJoMa9+EEXe7O0TlIeZuPsKvP9hAu5b+vHP7QMICfQls4U3nNpqclOdwaRWfiPxXRJJFZEs1/ceLSKaIbHD+HnFlPLUZ16MNLf19+GLD4ZMdY0dDp/Ng2QtQXOC+4JTHWLgzmXveXUd8VCif3TOK87pFEB8VqslJeRxXX4OaBVxcyzBLjDEDnL+/uDieGvn5eDO5bwe+23qU/KLSkz3G/Rayj8D6t90XnPIIpWWGJ+fuIDY8kHfvGE64vqpdeTCXJihjzGIg3ZXzqG9XDIgkt6iU/+04drJj3FiIGQlLn4eSQvcFp5q9OeuT2Hksmwcv6oG/r74JV3m2xtCKb6SIbBSRb0Skj7uDGR4XTtsQPz6vWM0nYktRWUnw7jVwbKv7AlTNVmFJKc/NS6BvVCiT4zu4Oxyl3M7dCWod0MkY0x94CZhT3YAicpeIrBGRNSkpKS4LyNtLuLx/JAt3JpOee7L1FJ0nwKXPwpFN8Op58M3D+kBZVa/eWXGQpIx8fntxD7z0oa9KuTdBGWOyjDE5zue5gK+IRFQz7H+MMUOMMUPatHHty9iuG9qRkjLDG0v3nuwoYh+BdN96GDwVVr4KS551aRzKMyRl5PPI51v427c7GN01nDHd9GWDSoGbm5mLSHvgmDHGiMgwbMJMc2dMAN3ahTC5bwdmLdvPHed1plVQi5M9A1vDpc/ZlxoueNze0NtlgvuCVU3amz/u57GvtiECPxkUzW8mdXd3SEo1Gq5uZj4bWA70EJFEEbldRO4WkfIbiq4BtojIRuBF4DpjjHFlTHV138Ru5BWX8nrFUlQ5Ebj8BWjTAz65HTKTGj5A1eQt3ZXKo19uZWz3Nix6aAJP/aQfbUP83R2WUo2GNJJ8cEaGDBli1qxZ4/L5/OK9dSzckczShyeeWooql5IAr02AllFw0ycQ1tHlMamma+muVDLzizm/V1tScwq5/KWltAnx47N7RhPkp/fMK9cSkbXGmCHujuNM6F5Rg/smdmPu5iO8tmQvv7245+kDtOkON3wAs6+HNybZJNWud8MHqhq9H/ekMnXmKkrKDMF+PoQG+FJSavj3zUM0OSlVDXe34mvUerQP4fJ+kby+dB+7k3OqHij2PJj2DZgymHkxrHtbW/epU+xLzWX6O+uIiwhi5rShTO7bHmMML1w3gLiIIHeHp1SjpVV8tUjOLuDC5xbTtW0wH/18ZPXNfzMOwse3Q+IqiBwIk5+F6MENEqNqvPal5nL7m6s5nlvE5784j5jwQHeHpDxUU6zi0xJULdqG+PPIZb1Ze+A4by3fX/2AYTFw+/dw9WuQfRT+exFsrfa2LtWMFRSX8vR3O5jwzEImPLOQxPR8Xr1psCYnpc6QJqg6uHpQFOO6t+Hv3+3kUHpe9QOKQL+fwj0rIGowfDQV1s5qqDBVI5CWU8hNr6/knwv20LF1II9e0Yf5D45jeOdwd4emVJOjCaoORIQnru6LMfC3b3fUPkJAGNz8GXS9AL68H/73GJSV1j6eatJ2J+dw9b9+ZHNSJv+8YRBv3TaMW0fFEt1KS05KnQ1NUHUUFRbAnWPi+GrTETYcyqh9hBaBcP1sGHgTLHkG3poC2cdqH081OWVlhlnL9nHZS0vIKShh9l0juLSfPktPqXOlCeoM3DWuC+FBLXhy7nbq1LjE2xem/NP+Ja6BV0fbKr/SEpfHquqfMYaC4lNLwpl5xdz0xkpmfLmNEZ3DmXv/GAbFtHJThEo1L5qgzkCwnw+/uqAbK/el87/tyXUfceBNcOd8aBVnq/xeGQELnoQFT8DCp+Bole9zVI3M419vZ9zTC0jLOfnKlae/38HKfek8dXVfZk4dSruW+iQIpeqLJqgzdN2wGDpHBPHkN9vJKzqDklC73raV33XvgZc3LHoKFv0NFj5pS1azr4ekda4LXJ2TPSk5zPxxP8eyCnl87nYAEo5l897Kg9w4PIbrhsUgok8gV6o+aYI6Q77eXsy4og97U3O5//0NlJadwX1kItDzUtvK75F0+HMGPLwfxv8BDi6H1y+ADbNdFrs6e09/uxN/Hy9uGB7Dp+uS+HF3Ko9/vZ0gPx9+dYE+4FUpV9AEdRbGdm/DI5f1Zt62YzzhnE2fERFbihKBgFYw/mG4f5N9KsWcu2HFq5CfAZs+gm//ADmue/+Vqt3aA+l8u/Uod43twiOX9aZTeCC/eG8dixJSuP/8brSu6jmNSqlzpgnqLE0bHcfUUbG8sXQf7648cO4T9G8JN34EPS+Dbx+Gp7vAp3fAin/Cuz+Bwuxzn4c6Y6Vlhifn7iAi2I87xsTh7+vNX6+M53heMbHhgdwyMtbdISrVbOlTKs/Bny7rzd7UXB77ahujukSc+3PVfPzg2jdh2fM2IfW4FPLT4f0b7d+NH9lhVINYuiuVJ+ZuZ9uRLJ64qu+Jh7qO6daGZ67tT+8OLWnho+d4SrmKPovvHB3LKuDC5xbRrV0IH/58JN6ueFX3htm26i96GHQeD+36QLdJ9l4rdVb+vWgPO49l8+y1/U9r3FBWZnjwo418uj6J6FYBPHRRD67oH6mNIFSTps/i80DtWvrz6JQ+rD1wnP8u3eeamQy4Hi57HvKP25t+P7oV3rzMfldnrKS0jNeW7OXTdUl8sfHwaf1f+CGBT9cncc/4Lvzwm3FMGRClyUkpN9AEVQ+uHBDFpN7tePr7nWw/kuWamQy5De5dA384DD95A45uhjcvh9xU18yvGVu1L53UnCJC/H3469fbySooPtHv601HeHH+bn46JJqHLuqBv6+3GyNVyrNpgqoHIsLjV/WlVaAvd7y55pQbOeudbwD0vQaufx9Sd8O/x8FbV8K718JH02Dub2Hx07BvCZQW1z49D/T15iME+Hrzxq1DSc0p5LnvE8gvKuXDNYd48KONDO7UiseujNdSk1Jupo0k6kmbED9eu2UI1766nOnvrOOdO4ZTZgxbD2fSq0NLAlvU86ruer59g+/iv0NRLpQWQuFuyE2Dwkw7jF+ove9q0mMQFGG7GQMFGbZ5uwcqKS3ju61HmdirLcPiWnPziE68tXw/n65LJKughJ7tQ/jXTYPw89GSk1LupgmqHvWLDuOZa/tz7+z1XPDcIo5k5lNcaji/Z1tev3VI/Z+Rx46G2M9P716QBfsWQcJ3sOkD2P0DXPkKePnYxyslroL4a2Dy0xDY2r4BOHkrRHRv9q0Ey6v3Lu1rH+b6wKQebE7KpGOrQG4YHsPwuNZaclKqkdAEVc8u7x9J4vF8fth+jEvi21NUWsbMZfuZveoQNwyPaZgg/FtCr8vt3/C74dM74d1rbL+WUTB4Kqx/B/YvgR6TbSLLPgx9roZrZzZMjG5SXr03oUdbAEIDfPnsntFujkopVRVNUC4wfXwXpo/vAtgmywnHsnnsq22M7BJ+7vdKnan28XDnAlj2AgSGw6BbbClpyG3w2XTY8J59b5X/eNj4nq0S7Osks/JbEJpJiaJi9V5AC63CU6qx0wTlYl5ewjPX9ufiF5bwy/fWcfe4LsRHhdKpdSBerrhnqiq+/jD+d6d269Afpi+zDSl8WthXgKTvga9/AzEjIGktfPt7MGUw9kEYeIsdrgl7Z8WBU6r3lFKNm96o20C+23qU+2avp7CkDIBRXcKZNW1Y43oSQfpe+Nd5NhHlH4d28dAiGA6tgNCO0LYX+AZCWAwMvR1axdrxjmyEvQttFWFYx5PTK8yxrQ69XF9aOZZVwMxl+/lu61EuiW/PvRO7nVJKem3xXh6fu50JPdrwn1uG4OvdiNa7Ug2gKd6oqwmqARWVlJFwLJtFCSk8/d1Obhwew+NX9XV3WKfaMBu++wOMecBev/Lyht3/g5WvQm6KbTF4fL8tWfW50r4l+MBSO66PP4z8JXQcBuvegp3f2AQVNQhix8KoX9rvZym7oJib31jFiM7hPDCpO77eXuQUlvD3b3cwe9VBSssM/aLD2HAo48QbkEucKtYP1yRyad8OPP+zAY3rpECpBqIJqoE01QRV0VPf7ODVRXt44qq+3DA8hoLiUjLziwlo4U2grzc+jfkMP+swLP8nrJlpWwEOu8s2e1/6PGz+yA4TGA79roPSIkhcDUc2QPu+8NO3oXXcWc32b9/u4F8L9wAwKCaM286L46lvdpCUkc/1w2L4+djOdAoPYuXeNP70+RYSjuUA4CX2PV6PTYl3zaOolGoCNEE1kOaQoErLDLfNWs2y3amEB7fgWNapN/fePa4Lv7ukp5uiq6PSYhCvU6vwDq+HrCM2YVVssp7wHXx6F2BgzIP2eYIto2zy2j3PjhMz3Ja0YoaDf+gps0rKyGfCMwu5tG8HJvZow/OfLUaKsilt3Y1nru3PkNjWpwxfUlrG0awCQvx9CfHzabjrfU1RUR6seMW+MPOKF0/eM1cbY+zJSmiUa+NrikpLIOeYbVHbItjWOORnQGGW3bb9w8CrYU9CNUE1kOaQoAAy84p5Yu52yowhulUgrYNbUFhcyvwdyaw9cJylD0+kTUgzui/p+H77tIvDld4cHNLBXtc6vN6WuBBo09NWFUYPgaghPPntTkp2L+TBbkcJSF5vd36geMS9+E561CbJ4nxI+Nb+Fy/nellHCI2x896/GJK322lHD4W2ve3Bwvsc2woZc3pLx+Tt0Lpz3e4rq2r8hlBSBBtn27c6Zx8B8bYnDrd+CQFhNY+blw6f/xJ2fg29roCLnjj1+uO5Ki6wJy++ARA12HXrxxjbIChlJ0QOgDa9oKzYJutDK+HYVkjeZhPK5S9ARDc7XsYhWP82HNlkhwGI7G/HT95mn+RSfsO8lw+UlQIVjrXibddXxxF2Oy8rtdto9mF74ldWaq/x9r/ONmiqh+XXBNVAmkuCqs6elBwueG4R94zvwkMXNfJS1Jkyxl7LSk2wO3n7vvagKGLP5BNXwaFVmEMrKTu4Gu+izFPHbxVrd+qoQfbAsO5N+2T3TqNttWNucs3zD253Irmd4B9qD7LjfmsTZW2Sd8C2z22iTd5mXyg54Ho479f2wPLd/9kDd6fRcMMH4BcCOcnwyR32wH/FS3aeRXnwxb32frRR99qm/y0q3IZQUghr/muftzjwRpvwjIHENfYNzEU5zlNEigED3i2g30/tAa2m9Z99BDZ/DCv+ZQ+IUUPs00aK8mD2dRA5EK78l12XuanQod/JBjElRbBvMXx5n12mvtfC1s9sv24X2pOD4jyIG2fvtwtpd3LepcU2EaTuBN8gWw0cFA7B7W0iStttTzB2zbPJoaTAjte+r61GDu9mD/Yidt2UFEDKDji4wo7b72f2uqmvP6TtgVWv2WdWHt9nE2qH/tBppC25F2bZ66cJ30DGwZMx+oXa6ZY6NRoto2zjoKR1tvtFT9jhV7xiT6bCu9n4TJmtxk7fa7ehzhPseivKtQ2OvFtAQGu7LRRkQl6q3QcOrrD7A9gTqpaR4O1nT7BSd9p5tO1tpxc1yP61ijurhKUJqoE09wQFcM+7a1myK5UffzeREH9fd4dTpYy8IoL9fOr9elnCsWz+/PlWlu9NJU6OMkB208offnPXHQS373rqwKvfgG9+C2Ul9lUko39lD6amzL5TK+Og/WvZAWLHQHBbe7BKXA3p++xjnzIO2Wtnpsze3Cxy8qAS0h6C2tgDb146HN0Ex7bYA0ibnvbg5d3CHvAx9szYy8feS7b+HXuwv/BR+PTnkJdmz85bd4bLXoDv/2hLjVGD7Fl8YAT0ucqWGHxawPy/2gOeeNnEEnuePcvOPHRy+X2DTjb/L8qzB9Y+V8GIe+zBzj/UziPhO/t0kbQ9NoEAxI2FUffb6tjyA962z+GjqXZdVNS6sy3pJq2Dknz7/Zr/2uXLOAg/zIDDG2yVFmKTt5evjbmk0L7XLH3fyQN/Zb6BJ+Nq0wu6TLDxZR+FVf+xJwLVCYuxsR1aaVubtu9rG+h4t3AO6LHg19Ku4yMb7LYCNhHEjbGtT6MG2fV0aJU9SYgZaf+Cwu2wWYdtFfX+JfZ7v5/B+Y9AaPSpsRQX2ARZV8bY9ecbYLezioknLx22fgpbPrWxlxTYdfv7QzbRnSFNUA3EExLUpsQMrnh5Gb+/pCc/H9fF3eGcJq+ohPFPL6RLm2DeuWN4nRofpOUU8uy8BIL9fHhgUvcTz7tLzSlk0c4UDmfksyclhy83HSHYz4d7J3YlLiIILy+hT2RL2oZUs+Mf2WgPqJEDz36BMhPtQ3Z3fmOvGQSE2bPX7KO2FOEbaJ9f2KoT9J4Cva88tXSQmWhLcCWF9r6xlpGw42t7sC8tgpBIuH62LfV8eItNVr5BTcI1OAAACjNJREFU8JPXoedkeya99HnYv9QOAxDRAy5+Atr2gbWzYMvHEN7VJqBuk06/jpGfActfhuWvQHHuqcvn3cIecNv1scml43B7hl+VgytsySS0o51H4mrYM9+e9UcPs9cIu006tbRXWeouW4I5sMxOI7CVTRQdBtjEXp60clPsOs5JhvAu0P2i00uxxtjEkp9hS6imzCZlbz/7e7SMtMPtWwzf/8km8aG329JUcNtTp1WUZ09c/EPPLJGAnffG9+1JSdSgMxv3XJUW22rj1ISTN9KfIU1QDcQTEhTATa+vZOexbL745Wg6hFbdPDuroJjFCSks2JFCl7ZBTB/XpUGeJTdr2T5mfGnPah+4sDv3nt/ttGFyCkvILiimqKSMFXvTePKbHeQUlFBSZujfMYwXrxvA/B3JPDcvgewCe1YbEdyCSX3a8+CkHrQOaiQ3BpeVnf0F7T0L7EHtwkdtaQzsGfPiZ2DoHacnibJSexDKTILO48D7LErPualw4EdbCsw/bg/8nSeAX/DZLYNqFjRBNRBPSVAr96Zx/WsrMMCw2NYMj2uNAYpKy0hMz2dXcjZ7U3IpKTME+/mQU1jCtYOjefLqvi5tpl5cWsb4pxfSPtSfjq0C+HLTET64awQxrQP5eF0iy/eksetYDkezCk4Zb2hsK564qi97UnJ46KNN5BSVYAyM6RbBby/qSbd2wfr+JaVcpCkmKJc+6khE/gtcBiQbY+Kr6C/AP4DJQB4w1RizrvJwnmp453B++M04vtx4hC82JvHi/N14Cfh4eREZ5k/XtiFM6t2e8f/f3t3HVnXXcRx/f9pSHspDC3QFS3lKa7XghA0YU4K4YQa4wIxzgks2Jwn+QRQWEwPhDx+iiXNGtyVzk4w9mQWWIU4CuqeKMfwxHgYLY1BGZxkUgYKljEGBQr/+cX5lN9AOWijnnnu/r+Sm95xzb/P95nt7vj2/87vnVBYzfngRT1Tv5fHqvTQ1t1BZ0o9t+49z7nwri6d/nikV0dThA42n2Vh7jGmVxR0elbXnv03NDOnfi5wcsX7HIQ42NfPz2WOYPHog2w808dBzWzjdcoELrUbV0P58pXwQ5Tf1pbB3Pvl5OQzum8/UimJyckRFST8qh/Tnsbc+YObYodw1psSvIO6cu0y3HkFJmgp8ArzYQYOaBfyIqEHdBjxuZrdd6fdmyxHUpVpb7Yrf53l2Yx2/XLeL3BzxxaH9ONHcwoHGZr5RVYIZVNccwQx65Io540qZN6mM8uJ+DOjT/lDSJ2fP8+v1u1i5+QC3jijiV/eM5eGX3+VCq/H64qnk5Ij36k+wZM0OppQP5rsTyxhd7ENJzqWbJB5BdfsQn6SRwLoOGtSfgH+Z2cqwvAeYZmaHPut3ZmuDuloNH5+hb688+uTncablAis21vHkhlp698hl3qThTK8q4dXtB1m1ZT9nWqLZWoMK8vlaZTHfmzScW0cU0XDyLJvrGvnt6zXUH2/mW+NL2VDTQFNzC2bw6L03850J1/F7L865bpXEBhX31cxLgZQ5s9SHdZc1KEkLgAUAw4ffoPsqJdRN/T+dndSrRy4Lv17O/CmjyJEuXoduXFkhi+6sYMu+Rvb97xQ1h0/yxvtHWLPtIP165V2ctDBiUB9e+eHtTBg5kOOnzvHIazXUHTvFnHF+9QDnXPeKu0FdNTNbDiyH6Agq5nASp73JB0UF0Yy5NqfOnmf9jkNs/aiRyiH9GVdWyJdKB1xsakUF+fzm2x1MTXbOuess7gZ1EEgdJxoW1rkYFPTM476JZdw30YfunHPxi/uS2WuBBxSZDJy40vkn55xz2aG7p5mvBKYBgyXVAz8DegCY2dPA34lm8NUSTTN/qDvjcc45lxzd2qDMbN4VthuwsDtjcM45l0xxD/E555xz7fIG5ZxzLi15g3LOOZeWvEE555xLS96gnHPOpaVE3m5D0lHgoy6+fTBw7DqGE7dMywcyL6dMywcyL6dMywcuz2mEmRXHFUxXJLJBXQtJW5N2wcTPkmn5QObllGn5QObllGn5QGbk5EN8zjnn0pI3KOecc2kpGxvU8rgDuM4yLR/IvJwyLR/IvJwyLR/IgJyy7hyUc865ZMjGIyjnnHMJ4A3KOedcWsqaBiVphqQ9kmolLYk7nq6QVCZpg6Rdkt6XtCisHyjpTUl7w8+iuGPtDEm5krZLWheWR0naFGr1sqT8uGPsDEmFklZLqpG0W9LtSa6RpIfD522npJWSeiWtRpKeldQgaWfKunZrEu5P90TIbYekW+KLvH0d5PNo+MztkPRXSYUp25aGfPZIuiueqDsvKxqUpFzgSWAmUAXMk1QVb1Rdch74iZlVAZOBhSGPJUC1mVUA1WE5SRYBu1OWHwH+YGblwHFgfixRdd3jwGtm9gXgy0S5JbJGkkqBHwMTzGwskAvMJXk1eh6Yccm6jmoyE6gIjwXAUzcoxs54nsvzeRMYa2Y3Ax8ASwHCPmIuMCa8549hn5j2sqJBAZOAWjP7j5mdA1YBc2KOqdPM7JCZbQvPTxLt+EqJcnkhvOwF4J54Iuw8ScOAbwLPhGUBdwCrw0uSls8AYCqwAsDMzplZEwmuEdF943pLygP6AIdIWI3M7N9A4yWrO6rJHOBFi7wNFEoaemMivTrt5WNmb5jZ+bD4NjAsPJ8DrDKzs2ZWR3SD2Ek3LNhrkC0NqhQ4kLJcH9YllqSRwHhgE1BiZofCpsNASUxhdcVjwE+B1rA8CGhK+UNLWq1GAUeB58Kw5TOSCkhojczsIPA7YD9RYzoBvEOya9Smo5pkwv7iB8A/wvPE5pMtDSqjSOoL/AVYbGYfp24LdylOxHcHJN0NNJjZO3HHch3lAbcAT5nZeOAUlwznJaxGRUT/gY8CPgcUcPnQUuIlqSZXImkZ0emAl+KO5VplS4M6CJSlLA8L6xJHUg+i5vSSma0Jq4+0DUGEnw1xxddJXwVmS9pHNOx6B9H5m8IwnATJq1U9UG9mm8LyaqKGldQaTQfqzOyombUAa4jqluQatemoJondX0j6PnA3cL99+iXXxOaTLQ1qC1ARZh7lE50wXBtzTJ0Wzs+sAHab2e9TNq0FHgzPHwT+dqNj6wozW2pmw8xsJFFN/mlm9wMbgHvDyxKTD4CZHQYOSKoMq+4EdpHQGhEN7U2W1Cd8/trySWyNUnRUk7XAA2E232TgRMpQYNqSNINouHy2mZ1O2bQWmCupp6RRRJM/NscRY6eZWVY8gFlEM1s+BJbFHU8Xc5hCNAyxA3g3PGYRnbepBvYCbwED4461C7lNA9aF56OJ/oBqgVeAnnHH18lcxgFbQ51eBYqSXCPgF0ANsBP4M9AzaTUCVhKdQ2shOsqd31FNABHN+v0QeI9oBmPsOVxFPrVE55ra9g1Pp7x+WchnDzAz7viv9uGXOnLOOZeWsmWIzznnXMJ4g3LOOZeWvEE555xLS96gnHPOpSVvUM4559KSNyjnnHNpyRuUc865tPR/NWOl3EKR/Z4AAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# plot both benchmarks\n",
        "plt.plot(testing_loss, label=\"no dropout\")\n",
        "plt.plot(testing_loss_d, label=\"with dropout\")\n",
        "# make the legend on the plot\n",
        "plt.legend()\n",
        "plt.title(\"The Cross-entropy loss of the MNIST test data with and w/o Dropout\")\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "_MazH3NNHenJ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 207
        },
        "outputId": "36467580-48bb-423c-9dd7-533996918bd7"
      },
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-15-3c10b8485208>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m   \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "import time\n",
        "\n",
        "while True:\n",
        "  time.sleep(30)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "DropoutRegularizationPyTorch_PythonCodeTutorial.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b4a6874638e2443f8c43e0db744fa327": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_befda00472714307afad3307ced59f39",
              "IPY_MODEL_073e2b2d0c164e5ba3ab91c64618e27c",
              "IPY_MODEL_367a2d97f6014f67b2557e5e26a76e2b"
            ],
            "layout": "IPY_MODEL_faa9f040562349cea6bf0b619249d5b4"
          }
        },
        "befda00472714307afad3307ced59f39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_506e3b6439b6476499deaf812300bf48",
            "placeholder": "​",
            "style": "IPY_MODEL_fee449ec39d24cbdbd6912bbdd95649f",
            "value": ""
          }
        },
        "073e2b2d0c164e5ba3ab91c64618e27c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ccda80a9207f470bb93b1ce9c7fb67d4",
            "max": 170498071,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ad67632afdf947c8b7136799037a63df",
            "value": 170498071
          }
        },
        "367a2d97f6014f67b2557e5e26a76e2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_50a6c0a6110e470291a56b008d7657b3",
            "placeholder": "​",
            "style": "IPY_MODEL_2ec3f8f6c8ae408ea62db2cc9368483a",
            "value": " 170499072/? [00:03&lt;00:00, 47975117.30it/s]"
          }
        },
        "faa9f040562349cea6bf0b619249d5b4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "506e3b6439b6476499deaf812300bf48": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fee449ec39d24cbdbd6912bbdd95649f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ccda80a9207f470bb93b1ce9c7fb67d4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ad67632afdf947c8b7136799037a63df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "50a6c0a6110e470291a56b008d7657b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2ec3f8f6c8ae408ea62db2cc9368483a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}